import sys
import gzip
import datetime
import threading
import math

# try to re-use this by Wei, 2012/10/05 evening.
# I think this is good program to re-use for building easy structure for clueweb2009 data.
# Let's measure the time to take as well.
# my default setting of the parameters will be 4 0/1/2/3 4 10000 using 4 processes, 4 threads each and make the last argument to be large like 10000

class ThreadClass(threading.Thread):
    STOP_THRESHOLD = -1
    NUMBER_OF_THREADS = -1
    processID = -1
    totalNumberOfProcesses = -1
    targetSegmentFileNameList = []
    existingDocIDWARCTRECIDPairDict = {}
    def run(self):
        print "Process",self.processID,self.getName(),"Created!","Each thread need to process at most",STOP_THRESHOLD,'compressed files.'
        
        
        threadSpecificTargetSegmentFileNameList = []
        numberOfSegmentsForEachThread = math.ceil( len(self.targetSegmentFileNameList)/float(self.NUMBER_OF_THREADS) )
        threadID = int( self.getName().strip().split("-")[1] )
        lowerBoundForSpecifcThread = int( (threadID-1)*numberOfSegmentsForEachThread )
        upperBoundForSpecificThread = int( lowerBoundForSpecifcThread + numberOfSegmentsForEachThread )
        for currentTargetSegmentFileName in self.targetSegmentFileNameList[lowerBoundForSpecifcThread:upperBoundForSpecificThread]:
            threadSpecificTargetSegmentFileNameList.append(currentTargetSegmentFileName)
        
        #print self.targetSegmentFileNameList
        #print threadSpecificTargetSegmentFileNameList
        
        compressedFileCounter = -1
        for targetSegmentFileName in threadSpecificTargetSegmentFileNameList:
            
            if compressedFileCounter == self.STOP_THRESHOLD:
                print "Stopping","Process",self.processID,self.getName(),"now!"
                break
            
            numberOfDocumentsAlreadyProcessedInThisThread = 0
            inputFileName = "/data1/team/weijiang/machine-learning-project-related/auxFiles/cluewebDataPaths/" + targetSegmentFileName
            inputFileHandler = open(inputFileName,'r')
            
            outputFileNameForDoc = "/data5/team/weijiang/the_new_trip_of_feature_generation/auxFiles/cluewebDataDocsMapping/" + targetSegmentFileName
            outputFileNameForDocInCompressedForm = "/data5/team/weijiang/the_new_trip_of_feature_generation/auxFiles/cluewebDataDocsMappingInCompressedForm/" + targetSegmentFileName
            outputFileNameForCompressedFile = "/data5/team/weijiang/the_new_trip_of_feature_generation/auxFiles/cluewebDataCompressedFileMapping/" + targetSegmentFileName
            
            outputFileForDocHandler = open(outputFileNameForDoc, "w")
            outputFileForDocCompressedFileHandler = gzip.GzipFile( outputFileNameForDocInCompressedForm + ".gz", "wb")
            
            outputFileForCompressedFileHandler = open(outputFileNameForCompressedFile, "w")
            
            #outputFileForCompressedFileHandler.write("Hello" + "\n")
            
            
            

            
            for currentLine in inputFileHandler.readlines():
                inputProcessingDataFileName = currentLine.strip()
                compressedFileCounter += 1
                
                if compressedFileCounter == -1:
                    print "Skipping.. ",inputProcessingDataFileName
                    continue
                elif compressedFileCounter == self.STOP_THRESHOLD:
                    
                    break
                else:
                    print compressedFileCounter,"Process",self.processID,self.getName(),"Processing ",inputProcessingDataFileName
                    outputFileForCompressedFileHandler.write(str(compressedFileCounter) + " " + inputProcessingDataFileName + "\n")
                inputProcessingDataFileHandler = gzip.GzipFile(inputProcessingDataFileName,'r')
                #inputProcessingDataFileHandler.readline().strip().split(":")[0] == "WARC-TREC-ID" or 
                
                
                absoluteDocBeginningPositionInOneCompressFile = 0
                absoluteDocEndingPositionInOneCompressFile = 0
                currentDataLine = inputProcessingDataFileHandler.readline()
                
                # the WARC header info for each compressed file, one time
                while currentDataLine:
                    if currentDataLine.strip().split(":")[0] == "Content-Length":
                        absoluteDocBeginningPositionInOneCompressFile = inputProcessingDataFileHandler.tell()
                        absoluteDocBeginningPositionInOneCompressFile += int(currentDataLine.strip().split(":")[1].strip() )
                        break
                    currentDataLine = inputProcessingDataFileHandler.readline()
        
                # the actual webpage content, many times
                flag1 = False
                flag2 = False
                flag3 = False
                while currentDataLine:
        
                    if currentDataLine.strip().split(":")[0] == "WARC-TREC-ID":
                        flag1 = True
                        currentWARCTRECID = currentDataLine.strip().split(":")[1].strip()
                        #print currentWARCTRECID
                        if currentWARCTRECID in self.existingDocIDWARCTRECIDPairDict:
                            flag3 = True
                            currentDocID = self.existingDocIDWARCTRECIDPairDict[currentWARCTRECID]
                        else:
                            flag3 = False
                    elif currentDataLine.strip().split(":")[0] == "Content-Length" and flag1 == True and flag2 == False:
                        flag2 = True
                        currentDocumentSizeInBytes = int(currentDataLine.strip().split(":")[1].strip() )
                        tempPosition = inputProcessingDataFileHandler.tell()
                        absoluteDocEndingPositionInOneCompressFile = tempPosition + currentDocumentSizeInBytes
                        if flag3:
                            numberOfDocumentsAlreadyProcessedInThisThread += 1
                            outputFileForDocHandler.write(currentDocID + " " + currentWARCTRECID + " " + str(compressedFileCounter) + " " + str(absoluteDocBeginningPositionInOneCompressFile) + " " + str(absoluteDocEndingPositionInOneCompressFile) + "\n")
                            outputFileForDocCompressedFileHandler.write(currentDocID + " " + currentWARCTRECID + " " + str(compressedFileCounter) + " " + str(absoluteDocBeginningPositionInOneCompressFile) + " " + str(absoluteDocEndingPositionInOneCompressFile) + "\n")
                        
                        #For each document, whether it is being recorded or not.
                        inputProcessingDataFileHandler.read( currentDocumentSizeInBytes )                
                        flag1 = False
                        flag2 = False
                        flag3 = False
                        absoluteDocBeginningPositionInOneCompressFile = absoluteDocEndingPositionInOneCompressFile
                    else:
                        pass
                    currentDataLine = inputProcessingDataFileHandler.readline()
                inputProcessingDataFileHandler.close()
                print "Total document recoreded:",numberOfDocumentsAlreadyProcessedInThisThread
                #flush these 3 files once the individual compressed file is finished.
                outputFileForDocHandler.flush()
                outputFileForDocCompressedFileHandler.flush()
                outputFileForCompressedFileHandler.flush()
                
            outputFileForDocHandler.close()
            outputFileForDocCompressedFileHandler.close()
            outputFileForCompressedFileHandler.close()


'''  
def buildIndexTry1(existingDocIDWARCTRECIDPairDict):
    print "This method currently is NOT use. Thanks"
    number_of_documents_needed_to_fix = len(existingDocIDWARCTRECIDPairDict)
    
    inputFileName = "/data1/team/weijiang/machine-learning-project-related/programs/polyIRToolkit/polyIRIndexer/clueWebOffcialIndexPathsALL.txt"
    inputFileHandler = open(inputFileName,'r')
    
    outputFileNameForDoc = "/data1/team/weijiang/machine-learning-project-related/auxFiles/docsMapping"
    outputFileNameForCompressedFile = "/data1/team/weijiang/machine-learning-project-related/auxFiles/compressedFileMapping"
    
    outputFileForDocHandler = open(outputFileNameForDoc, "w")
    outputFileForCompressedFileHandler = open(outputFileNameForCompressedFile, "w")
    outputCompressedFileForCompressedFileHandler = gzip.GzipFile( outputFileNameForDoc + ".gz", "wb")
    
    
    # read the file, uncompress them and build the index table.
    currentInParsingDocumentFlag = False
    compressedFileID = -1
    #currentDocID = 0
    currentOutputLine = ""
    #BeginningDocID = -1
    #EndingDocID = -1
    statusLine = ""
    
    print "build index()"
    print "number_of_documents_needed_to_fix:",number_of_documents_needed_to_fix
    #print "length:",len(existingDocIDWARCTRECIDPairDict)
    
    #for element in existingDocIDWARCTRECIDPairDict:
    #    print element,existingDocIDWARCTRECIDPairDict[element]
    #testString = raw_input("pause:")
    
    print "in"
    for currentLine in inputFileHandler.readlines():
        currentInParsingDocumentFlag = False
        absoluteDocBeginningPositionInOneCompressFile = 0
        absoluteDocEndingPositionInOneCompressFile = 0
        #BeginningDocID = currentDocID
        
        #print "compressedFileID:",compressedFileID
        compressedFileID += 1
        if compressedFileID == 0:
            continue
        elif compressedFileID == 2:
            break
        
        dataInputFileName = currentLine.strip()
        print "Processing:",dataInputFileName
        outputFileForCompressedFileHandler.write(str(compressedFileID) + " ")
        outputFileForCompressedFileHandler.write(dataInputFileName + " ")
        dataInputFileHandler = gzip.GzipFile(dataInputFileName,'r')
        #print dataInputFileHandler.read()
        #print len(dataInputFileHandler.readlines())
        #print dataInputFileHandler.readlines()
        #print dataInputFileHandler.readline()
        
        
        
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        
        currentWARCHeaderSizeInBytes = int(dataInputFileHandler.readline().strip().split(":")[1].strip() )
        absoluteDocBeginningPositionInOneCompressFile = dataInputFileHandler.tell()
        absoluteDocBeginningPositionInOneCompressFile += currentWARCHeaderSizeInBytes
        #print "*",dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
    
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        dataInputFileHandler.readline()
        # record the DocID
        # print currentDocID
        #currentOutputLine = str(currentDocID) + " "
        dataInputFileHandler.readline().strip()
        # record the WARCTRECID
        currentWARCTRECID = dataInputFileHandler.readline().strip().split(":")[1].strip()
        
        if currentWARCTRECID in existingDocIDWARCTRECIDPairDict:
            currentInParsingDocumentFlag = True
            currentOutputLine = existingDocIDWARCTRECIDPairDict[currentWARCTRECID] + " "
            currentOutputLine += currentWARCTRECID + " "
        else:
            
            currentInParsingDocumentFlag = False
        
        dataInputFileHandler.readline().strip()
        dataInputFileHandler.readline().strip()
        currentDocSizeInBytes = int(dataInputFileHandler.readline().strip().split(":")[1].strip() )
        absoluteDocEndingPositionInOneCompressFile = dataInputFileHandler.tell()
        absoluteDocEndingPositionInOneCompressFile += currentDocSizeInBytes
        # record the absolutePosition
        if currentInParsingDocumentFlag:
            currentOutputLine += str(compressedFileID) + " "
            currentOutputLine += str(absoluteDocBeginningPositionInOneCompressFile) + " "
            currentOutputLine += str(absoluteDocEndingPositionInOneCompressFile) + " "
            print currentOutputLine.strip()
            outputFileForDocHandler.write( currentOutputLine.strip() + "\n")
            outputCompressedFileForCompressedFileHandler.write( currentOutputLine.strip() + "\n" )
            number_of_documents_needed_to_fix -= 1
            
        else:
            #we just skip that
            pass
        #outputCompressedFileForCompressedFileHandler.write( currentOutputLine.strip() + "\n" )
        
        #currentDocID += 1
        dataInputFileHandler.read(currentDocSizeInBytes)
        
        
        
        while True:
            currentInParsingDocumentFlag = False
            absoluteDocBeginningPositionInOneCompressFile = absoluteDocEndingPositionInOneCompressFile
            if not dataInputFileHandler.readline(): break
            if not dataInputFileHandler.readline(): break
            if not dataInputFileHandler.readline(): break
            if not dataInputFileHandler.readline(): break
            if not dataInputFileHandler.readline(): break
            if not dataInputFileHandler.readline(): break
            
            # record the DocID
            # print currentDocID
            #currentOutputLine = str(currentDocID) + " "
            
            if not dataInputFileHandler.readline(): break
            
            # record the WARCTRECID
            currentWARCTRECID = dataInputFileHandler.readline().strip().split(":")[1].strip()
            
            
            if currentWARCTRECID in existingDocIDWARCTRECIDPairDict:
                
                currentInParsingDocumentFlag = True
                currentOutputLine = existingDocIDWARCTRECIDPairDict[currentWARCTRECID] + " "
                currentOutputLine += currentWARCTRECID + " "
            else:
                
                currentInParsingDocumentFlag = False
            # for test
            if existingDocIDWARCTRECIDPairDict[currentWARCTRECID] != "266307":
                if not dataInputFileHandler.readline(): break
                if not dataInputFileHandler.readline(): break
            else:
                print dataInputFileHandler.readline()
                print dataInputFileHandler.readline()
            
            try:
                tempString = dataInputFileHandler.readline()
                currentDocSizeInBytes = int( tempString.strip().split(":")[1].strip() )
            except ValueError:
                print "Oops:",tempString,currentWARCTRECID
            absoluteDocEndingPositionInOneCompressFile += dataInputFileHandler.tell() - absoluteDocBeginningPositionInOneCompressFile
            absoluteDocEndingPositionInOneCompressFile += currentDocSizeInBytes
            
            # record the absolutePosition
            if currentInParsingDocumentFlag:
                currentOutputLine += str(compressedFileID) + " "
                currentOutputLine += str(absoluteDocBeginningPositionInOneCompressFile) + " "
                currentOutputLine += str(absoluteDocEndingPositionInOneCompressFile) + " "
                print currentOutputLine.strip()
                outputFileForDocHandler.write( currentOutputLine.strip() + "\n")
                outputCompressedFileForCompressedFileHandler.write( currentOutputLine.strip() + "\n" )
                number_of_documents_needed_to_fix -= 1
                
            else:
                #we just skip that
                pass
            #outputCompressedFileForCompressedFileHandler.write( currentOutputLine.strip() + "\n" )
            
            #currentDocID += 1
            dataInputFileHandler.read(currentDocSizeInBytes)    
    
        
        #but not include in the current compressed file
        #EndingDocID = currentDocID
        #statusLine = "[" + str(BeginningDocID) + "," + str(EndingDocID) + ")"
        print statusLine
        #outputFileForCompressedFileHandler.write(str(BeginningDocID) + " ")
        #outputFileForCompressedFileHandler.write(str(EndingDocID) + " ")
        outputFileForCompressedFileHandler.write("\n")
        
        #if compressedFileID == 2:
        #    break;
        print "number_of_documents_needed_to_fix:",number_of_documents_needed_to_fix
    outputFileForDocHandler.close()
    outputFileForCompressedFileHandler.close()
    #outputCompressedFileForCompressedFileHandler.close()
    print "out"
    print "number_of_documents_needed_to_fix:",number_of_documents_needed_to_fix
'''

'''
def loadIndex():
    docIDPositionPairDict = {}
    compressedFileIDPathDict = {}
    
    inputFileNameForDocsMapping = "/data1/team/weijiang/machine-learning-project-related/auxFiles/docsMapping.gz"
    inputFileNameForCompressedFileMapping = "/data1/team/weijiang/machine-learning-project-related/auxFiles/compressedFileMapping"
    inputFileForDocsMappingHandler = gzip.GzipFile( inputFileNameForDocsMapping, "rb")
    for currentLine in inputFileForDocsMappingHandler.readlines():
        if len( currentLine.strip().split(" ") ) == 5:
            currentDocID = int(currentLine.strip().split(" ")[0])
            #currently not used.
            currentWRACTRECID = currentLine.strip().split(" ")[1]
            currentDocBelongingCompressedFile = currentLine.strip().split(" ")[2]
            currentBeginningPosition = currentLine.strip().split(" ")[3]
            currentEndingPosition = currentLine.strip().split(" ")[4]
            docIDPositionPairDict[currentDocID] = currentDocBelongingCompressedFile + "_" + currentBeginningPosition + "_" + currentEndingPosition
        else:
            print "problem"
            sys.exit() 
    inputFileForCompressedFileHandler = open(inputFileNameForCompressedFileMapping,"r")
    for currentLine in inputFileForCompressedFileHandler.readlines():
        #print "currentLine:",currentLine
        if len(currentLine.strip().split(" ") ) == 2:
            currentUniqueCompressedFileNameID = int( currentLine.strip().split(" ")[0] )
            currentCompressedFileNamePath = currentLine.strip().split(" ")[1]
            
            compressedFileIDPathDict[currentUniqueCompressedFileNameID] = currentCompressedFileNamePath
    
    print len(docIDPositionPairDict),"Loaded."
    print len(compressedFileIDPathDict),"Loaded."
    
    
    
    while True:
        inputString = raw_input('Enter DocID to retrieve the actual document content:')
        inputDocID = int(inputString.strip())
        if len(docIDPositionPairDict[inputDocID].strip().split("_")) == 3:
            compressedFileID = int(docIDPositionPairDict[inputDocID].strip().split("_")[0])
            beginningPosition = int(docIDPositionPairDict[inputDocID].strip().split("_")[1])
            endingPosition = int(docIDPositionPairDict[inputDocID].strip().split("_")[2])
            
            targetFileNamePath = compressedFileIDPathDict[compressedFileID]
            targetFileHandler = gzip.GzipFile(targetFileNamePath, "r")
            targetFileHandler.seek(beginningPosition)
            targetWebPageContent = targetFileHandler.read(endingPosition - beginningPosition)
            print targetWebPageContent
            targetFileHandler.close()
        else:
            print "problem2"
    pass
'''
'''
def buildIndexTry2(existingDocIDWARCTRECIDPairDict):
    print "This method currently is being use. Thanks"
    print "Let's make it into multi-threads"
    
    numberOfDocumentsAlreadyProcessed = 0
    print "buildIndexTry2"
    number_of_documents_needed_to_fix = len(existingDocIDWARCTRECIDPairDict)
    print "number_of_documents_needed_to_fix:",number_of_documents_needed_to_fix
    
    inputFileName = "/data1/team/weijiang/machine-learning-project-related/programs/polyIRToolkit/polyIRIndexer/clueWebOffcialIndexPathsALL.txt"
    inputFileHandler = open(inputFileName,'r')
    
    outputFileNameForDoc = "/data1/team/weijiang/machine-learning-project-related/auxFiles/docsMapping"
    outputFileNameForCompressedFile = "/data1/team/weijiang/machine-learning-project-related/auxFiles/compressedFileMapping"
    
    outputFileForDocHandler = open(outputFileNameForDoc, "w")
    outputFileForDocCompressedFileHandler = gzip.GzipFile( outputFileNameForDoc + ".gz", "wb")
    
    outputFileForCompressedFileHandler = open(outputFileNameForCompressedFile, "w")
    
    #outputFileForCompressedFileHandler.write("Hello" + "\n")
    
    
    
    compressedFileCounter = -2
    for currentLine in inputFileHandler.readlines():
        inputProcessingDataFileName = currentLine.strip()
        compressedFileCounter += 1
        
        if compressedFileCounter == -1:
            print "Skipping.. ",inputProcessingDataFileName
            continue
        
        #elif compressedFileCounter == 10:
        #    print "Stopping.. "
        #    break
        else:
            print "Processing ",inputProcessingDataFileName
            outputFileForCompressedFileHandler.write(str(compressedFileCounter) + " " + inputProcessingDataFileName + "\n")
        inputProcessingDataFileHandler = gzip.GzipFile(inputProcessingDataFileName,'r')
        #inputProcessingDataFileHandler.readline().strip().split(":")[0] == "WARC-TREC-ID" or 
        
        
        absoluteDocBeginningPositionInOneCompressFile = 0
        absoluteDocEndingPositionInOneCompressFile = 0
        currentDataLine = inputProcessingDataFileHandler.readline()
        while currentDataLine:
            if currentDataLine.strip().split(":")[0] == "Content-Length":
                absoluteDocBeginningPositionInOneCompressFile = inputProcessingDataFileHandler.tell()
                absoluteDocBeginningPositionInOneCompressFile += int(currentDataLine.strip().split(":")[1].strip() )
                break
            currentDataLine = inputProcessingDataFileHandler.readline()

        flag1 = False
        flag2 = False
        flag3 = False
        while currentDataLine:

            if currentDataLine.strip().split(":")[0] == "WARC-TREC-ID":
                flag1 = True
                currentWARCTRECID = currentDataLine.strip().split(":")[1].strip()
                #print currentWARCTRECID
                if currentWARCTRECID in existingDocIDWARCTRECIDPairDict:
                    flag3 = True
                    currentDocID = existingDocIDWARCTRECIDPairDict[currentWARCTRECID]
                else:
                    flag3 = False
            elif currentDataLine.strip().split(":")[0] == "Content-Length" and flag1 == True and flag2 == False:
                flag2 = True
                currentDocumentSizeInBytes = int(currentDataLine.strip().split(":")[1].strip() )
                tempPosition = inputProcessingDataFileHandler.tell()
                absoluteDocEndingPositionInOneCompressFile = tempPosition + currentDocumentSizeInBytes
                if flag3:
                    numberOfDocumentsAlreadyProcessed += 1
                    outputFileForDocHandler.write(currentDocID + " " + currentWARCTRECID + " " + str(compressedFileCounter) + " " + str(absoluteDocBeginningPositionInOneCompressFile) + " " + str(absoluteDocEndingPositionInOneCompressFile) + "\n")
                    outputFileForDocCompressedFileHandler.write(currentDocID + " " + currentWARCTRECID + " " + str(compressedFileCounter) + " " + str(absoluteDocBeginningPositionInOneCompressFile) + " " + str(absoluteDocEndingPositionInOneCompressFile) + "\n")
                
                #For each document, whether it is being recorded or not.
                inputProcessingDataFileHandler.read( currentDocumentSizeInBytes )                
                flag1 = False
                flag2 = False
                flag3 = False
                absoluteDocBeginningPositionInOneCompressFile = absoluteDocEndingPositionInOneCompressFile
            else:
                pass
            currentDataLine = inputProcessingDataFileHandler.readline()
        inputProcessingDataFileHandler.close()
        print "numberOfDocumentsAlreadyProcessed:",numberOfDocumentsAlreadyProcessed
    outputFileForDocHandler.flush()
    outputFileForDocHandler.close()
    
    outputFileForDocCompressedFileHandler.flush()
    outputFileForDocCompressedFileHandler.close()
    
    outputFileForCompressedFileHandler.flush()
    outputFileForCompressedFileHandler.close()
'''

'''
def retreiveInfoFromExistingDataset():
    print "retreive Info From Existing Dataset"
    docIDWARCTRECIDPairDirct = {}
    inputFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/auxFiles/docID_WARCTRECID_1M_pair"
    inputFileHandler = open(inputFileName, "r")
    passCounter = 0
    for currentLine in inputFileHandler.readlines():
        if currentLine.strip().split(" ")[1] not in docIDWARCTRECIDPairDirct:
            docIDWARCTRECIDPairDirct[ currentLine.strip().split(" ")[1] ] =  currentLine.strip().split(" ")[0]
        else:
            passCounter += 1
    print "len(docIDWARCTRECIDPairDirct)",len(docIDWARCTRECIDPairDirct)
    print "passCounter",passCounter
    
    return docIDWARCTRECIDPairDirct
'''



def loadDocIDWARCTRECID1MPair():
    # This method is NOT used any more due to it is limited to certain small dataset.
    existingDocIDWARCTRECIDPairDict = {}
    inputFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/auxFiles/docID_WARCTRECID_1M_pair"
    inputFileHandler = open(inputFileName, "r")
    passCounter = 0
        
    for currentLine in inputFileHandler.readlines():
        if currentLine.strip().split(" ")[1] not in existingDocIDWARCTRECIDPairDict:
            existingDocIDWARCTRECIDPairDict[ currentLine.strip().split(" ")[1] ] =  currentLine.strip().split(" ")[0]
        else:
            passCounter += 1
        
    #print "len(existingDocIDWARCTRECIDPairDict)",len(existingDocIDWARCTRECIDPairDict)
    #print "passCounter",passCounter
    inputFileHandler.close()
    return existingDocIDWARCTRECIDPairDict


def loadTrecIDWhichNeedToBuildEasyAccessTo():
    # This method is currently being used.
    existingDocIDWARCTRECIDPairDict = {}
    inputFileName = "/data1/team/weijiang/machine-learning-project-related/trec-related/trecCombine09_11/priorityWarcDocumentsIDs_all_sorted_with_index_number.txt"
    inputFileHandler = open(inputFileName, "r")
    passCounter = 0
        
    for currentLine in inputFileHandler.readlines():
        if currentLine.strip().split(" ")[1] not in existingDocIDWARCTRECIDPairDict:
            # Just put the TrecID index number into the dict instead of actual docID number or sth.
            existingDocIDWARCTRECIDPairDict[ currentLine.strip().split(" ")[1] ] =  currentLine.strip().split(" ")[0]
        else:
            passCounter += 1
        
    #print "len(existingDocIDWARCTRECIDPairDict)",len(existingDocIDWARCTRECIDPairDict)
    #print "passCounter",passCounter
    inputFileHandler.close()
    return existingDocIDWARCTRECIDPairDict

def loadTargetsSegmentNameTable():
    #step0: read the segment names.
    segmentNameList = []
    inputFileNameForThreadPathMapping = "/data1/team/weijiang/machine-learning-project-related/auxFiles/threadPathMappingTable"
    inputFileHandlerForThreadPathMapping = open(inputFileNameForThreadPathMapping, "r")
    for currentLine in inputFileHandlerForThreadPathMapping.readlines():
        segmentNameList.append( currentLine.strip() )
    inputFileHandlerForThreadPathMapping.close()
    return segmentNameList

if len(sys.argv) < 5 :
    print "Usage: python scriptForBuildingEasyStructureForCluewebData.py <total # of processes> <current process ID> <# of threads> <# of compressed files needed to process>"
    sys.exit()
elif len(sys.argv) == 5:
    STOP_THRESHOLD = int(sys.argv[4])
    NUMBER_OF_THREADS = int(sys.argv[3])
    
    segmentNameList = loadTargetsSegmentNameTable()
    
    numberOfSegmentsForEachProcess = math.ceil( len(segmentNameList)/float(sys.argv[1]) )
    #print "numberOfSegmentsForEachProcess:",numberOfSegmentsForEachProcess

    #working old version
    
    for i in range(NUMBER_OF_THREADS):
        t = ThreadClass()
        t.STOP_THRESHOLD = STOP_THRESHOLD
        t.NUMBER_OF_THREADS = NUMBER_OF_THREADS
        t.totalNumberOfProcesses = int(sys.argv[1])
        t.processID = int(sys.argv[2])
        
        # Wei: 2012/10/05: The loadDocIDWARCTRECID1MPair() needed to be replaced.
        # The new replacement will be loadTrecIDWhichNeedToBuildEasyAccessTo()
        t.existingDocIDWARCTRECIDPairDict = loadTrecIDWhichNeedToBuildEasyAccessTo()
        
        #Let's assign the segment names based on the processID
        lowerBound = int( t.processID*numberOfSegmentsForEachProcess )
        upperBound = int( lowerBound + numberOfSegmentsForEachProcess )
        #print "lowerBound:",lowerBound
        #print "upperBound:",upperBound
        for currentSegmentName in segmentNameList[lowerBound:upperBound]:
            if currentSegmentName not in t.targetSegmentFileNameList:
                t.targetSegmentFileNameList.append(currentSegmentName)
            else:
                pass
        t.start()    
    #print "Run the Process", t.processID,"in",NUMBER_OF_THREADS,"Threads Done!"
else:
    print "Unknown Command"
    sys.exit() 
    


#dict1 = retreiveInfoFromExistingDataset()
#buildIndexTry2(dict1)
#loadIndex()

